{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor, StackingRegressor\n",
    "from lightgbm import LGBMRegressor\n",
    "from xgboost import XGBRegressor, XGBRFRegressor\n",
    "from catboost import CatBoostRegressor\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import mean_squared_error, make_scorer\n",
    "\n",
    "from bayes_opt import BayesianOptimization\n",
    "from bayes_opt.util import Colours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing files\n",
    "train = pd.read_csv('/run/media/devesh/Hard Disk/Projects/Machinehack-Melanoma Tumor Size Predicition/input/Train.csv')\n",
    "test = pd.read_csv('/run/media/devesh/Hard Disk/Projects/Machinehack-Melanoma Tumor Size Predicition/input/Test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mass_npea</th>\n",
       "      <th>size_npear</th>\n",
       "      <th>malign_ratio</th>\n",
       "      <th>damage_size</th>\n",
       "      <th>exposed_area</th>\n",
       "      <th>std_dev_malign</th>\n",
       "      <th>err_malign</th>\n",
       "      <th>malign_penalty</th>\n",
       "      <th>damage_ratio</th>\n",
       "      <th>tumor_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6930.90</td>\n",
       "      <td>2919.02</td>\n",
       "      <td>0.42116</td>\n",
       "      <td>51.8298</td>\n",
       "      <td>9.888294e+05</td>\n",
       "      <td>109.487</td>\n",
       "      <td>2758.76</td>\n",
       "      <td>72</td>\n",
       "      <td>39.3620</td>\n",
       "      <td>14.103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15635.70</td>\n",
       "      <td>4879.36</td>\n",
       "      <td>0.31206</td>\n",
       "      <td>223.5500</td>\n",
       "      <td>2.058426e+06</td>\n",
       "      <td>248.881</td>\n",
       "      <td>5952.53</td>\n",
       "      <td>240</td>\n",
       "      <td>22.0253</td>\n",
       "      <td>2.648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10376.20</td>\n",
       "      <td>2613.88</td>\n",
       "      <td>0.25191</td>\n",
       "      <td>127.3370</td>\n",
       "      <td>1.434676e+06</td>\n",
       "      <td>160.093</td>\n",
       "      <td>4635.26</td>\n",
       "      <td>73</td>\n",
       "      <td>29.9963</td>\n",
       "      <td>1.688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13093.80</td>\n",
       "      <td>4510.06</td>\n",
       "      <td>0.34444</td>\n",
       "      <td>155.4400</td>\n",
       "      <td>1.812195e+06</td>\n",
       "      <td>173.015</td>\n",
       "      <td>5273.87</td>\n",
       "      <td>32</td>\n",
       "      <td>28.1354</td>\n",
       "      <td>3.796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7545.21</td>\n",
       "      <td>2882.36</td>\n",
       "      <td>0.38201</td>\n",
       "      <td>85.1237</td>\n",
       "      <td>1.043918e+06</td>\n",
       "      <td>124.414</td>\n",
       "      <td>3263.35</td>\n",
       "      <td>57</td>\n",
       "      <td>35.0200</td>\n",
       "      <td>18.023</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mass_npea  size_npear  malign_ratio  damage_size  exposed_area  \\\n",
       "0    6930.90     2919.02       0.42116      51.8298  9.888294e+05   \n",
       "1   15635.70     4879.36       0.31206     223.5500  2.058426e+06   \n",
       "2   10376.20     2613.88       0.25191     127.3370  1.434676e+06   \n",
       "3   13093.80     4510.06       0.34444     155.4400  1.812195e+06   \n",
       "4    7545.21     2882.36       0.38201      85.1237  1.043918e+06   \n",
       "\n",
       "   std_dev_malign  err_malign  malign_penalty  damage_ratio  tumor_size  \n",
       "0         109.487     2758.76              72       39.3620      14.103  \n",
       "1         248.881     5952.53             240       22.0253       2.648  \n",
       "2         160.093     4635.26              73       29.9963       1.688  \n",
       "3         173.015     5273.87              32       28.1354       3.796  \n",
       "4         124.414     3263.35              57       35.0200      18.023  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = train.pop('tumor_size')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature engineering\n",
    "\n",
    "train['penalty-err'] = train['malign_penalty'] - train['err_malign']\n",
    "train['d_size-ratio'] = train['damage_size'] - train['damage_ratio']\n",
    "train['d_ratio-m_ratio'] = train['damage_ratio'] - (train['malign_ratio'])\n",
    "train['penalty/std'] = train['malign_penalty']/train['std_dev_malign']\n",
    "train['mass/area'] = (train['mass_npea'])/(train['exposed_area'])\n",
    "train['area/mass'] = train['exposed_area']/train['mass_npea']\n",
    "train['err/std'] = train['penalty-err']/train['std_dev_malign']\n",
    "train['dsr/ps'] = train['damage_size']/train['penalty-err']\n",
    "train['std/area'] = train['std_dev_malign']/train['exposed_area']\n",
    "train['err/area'] = train['err_malign']/train['exposed_area']\n",
    "train['dr/area'] = (train['damage_ratio']*100)/train['exposed_area']\n",
    "train['std/err'] = (train['std_dev_malign']+1)/(train['err_malign']+1)\n",
    "train['penalty/err'] = (train['malign_penalty']+1)/(train['err_malign']+1)\n",
    "\n",
    "test['penalty-err'] = test['malign_penalty'] - test['err_malign']\n",
    "test['d_size-ratio'] = test['damage_size'] - test['damage_ratio']\n",
    "test['d_ratio-m_ratio'] = test['damage_ratio'] - (test['malign_ratio'])\n",
    "test['penalty/std'] = test['malign_penalty']/test['std_dev_malign']\n",
    "test['mass/area'] = (test['mass_npea'])/(test['exposed_area'])\n",
    "test['area/mass'] = test['exposed_area']/test['mass_npea']\n",
    "test['err/std'] = test['penalty-err']/test['std_dev_malign']\n",
    "test['dsr/ps'] = test['damage_size']/test['penalty-err']\n",
    "test['std/area'] = test['std_dev_malign']/test['exposed_area']\n",
    "test['err/area'] = test['err_malign']/test['exposed_area']\n",
    "test['dr/area'] = (test['damage_ratio']*100)/test['exposed_area']\n",
    "test['std/err'] = (test['std_dev_malign']+1)/(test['err_malign']+1)\n",
    "test['penalty/err'] = (test['malign_penalty']+1)/(test['err_malign']+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['mass_npea', 'size_npear', 'malign_ratio', 'damage_size',\n",
       "       'exposed_area', 'std_dev_malign', 'err_malign', 'malign_penalty',\n",
       "       'damage_ratio', 'penalty-err', 'd_size-ratio', 'd_ratio-m_ratio',\n",
       "       'penalty/std', 'mass/area', 'area/mass', 'err/std', 'dsr/ps',\n",
       "       'std/area', 'err/area', 'dr/area', 'std/err', 'penalty/err'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['mass_npea', 'size_npear', 'malign_ratio', 'damage_size',\n",
       "       'exposed_area', 'std_dev_malign', 'err_malign', 'malign_penalty',\n",
       "       'damage_ratio', 'penalty-err', 'd_size-ratio', 'd_ratio-m_ratio',\n",
       "       'penalty/std', 'mass/area', 'area/mass', 'err/std', 'dsr/ps',\n",
       "       'std/area', 'err/area', 'dr/area', 'std/err', 'penalty/err'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformation\n",
    "\n",
    "train['exposed_area'] = np.log1p(train['exposed_area'])\n",
    "\n",
    "test['exposed_area'] = np.log1p(test['exposed_area'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#defining metric\n",
    "\n",
    "def RMSE(y_true, y_pred):\n",
    "    return np.sqrt(mean_squared_error(y_true, y_pred))\n",
    "\n",
    "rmse = make_scorer(RMSE, greater_is_better=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyperparameter tuning of extratrees Regressor using Bayesian optimization\n",
    "\n",
    "def etc_cv(n_estimators, min_samples_split, max_features, data, targets):\n",
    "    estimator = ExtraTreesRegressor(\n",
    "        n_estimators=n_estimators,\n",
    "        min_samples_split=min_samples_split,\n",
    "        max_features=max_features,\n",
    "        random_state=2,\n",
    "        n_jobs = -1\n",
    "    )\n",
    "    cval = cross_val_score(estimator, data, targets,\n",
    "                           scoring=rmse, cv=5)\n",
    "    return cval.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_etc(data, targets):\n",
    "    def etc_crossval(n_estimators, min_samples_split, max_features):\n",
    "        return etc_cv(\n",
    "            n_estimators=int(n_estimators),\n",
    "            min_samples_split=int(min_samples_split),\n",
    "            max_features=max(min(max_features, 0.999), 1e-3),\n",
    "            data=data,\n",
    "            targets=targets,\n",
    "        )\n",
    "\n",
    "    optimizer = BayesianOptimization(\n",
    "        f=etc_crossval,\n",
    "        pbounds={\n",
    "            \"n_estimators\": (100, 400),\n",
    "            \"min_samples_split\": (2, 25),\n",
    "            \"max_features\": (0.1, 0.999),\n",
    "        },\n",
    "        random_state=1234,\n",
    "        verbose=2\n",
    "    )\n",
    "    optimizer.maximize(n_iter=15, init_points=10)\n",
    "\n",
    "    print(\"Final result:\", optimizer.max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[93m--- Optimizing Extra Trees ---\u001b[0m\n",
      "|   iter    |  target   | max_fe... | min_sa... | n_esti... |\n",
      "-------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m-3.953   \u001b[0m | \u001b[0m 0.2722  \u001b[0m | \u001b[0m 16.31   \u001b[0m | \u001b[0m 231.3   \u001b[0m |\n",
      "| \u001b[95m 2       \u001b[0m | \u001b[95m-3.906   \u001b[0m | \u001b[95m 0.806   \u001b[0m | \u001b[95m 19.94   \u001b[0m | \u001b[95m 181.8   \u001b[0m |\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m-3.97    \u001b[0m | \u001b[0m 0.3485  \u001b[0m | \u001b[0m 20.44   \u001b[0m | \u001b[0m 387.4   \u001b[0m |\n",
      "| \u001b[95m 4       \u001b[0m | \u001b[95m-3.767   \u001b[0m | \u001b[95m 0.8875  \u001b[0m | \u001b[95m 10.23   \u001b[0m | \u001b[95m 250.3   \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m-3.893   \u001b[0m | \u001b[0m 0.7144  \u001b[0m | \u001b[0m 18.39   \u001b[0m | \u001b[0m 211.1   \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m-3.824   \u001b[0m | \u001b[0m 0.6045  \u001b[0m | \u001b[0m 13.57   \u001b[0m | \u001b[0m 104.1   \u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m-3.944   \u001b[0m | \u001b[0m 0.7948  \u001b[0m | \u001b[0m 22.3    \u001b[0m | \u001b[0m 209.5   \u001b[0m |\n",
      "| \u001b[95m 8       \u001b[0m | \u001b[95m-3.668   \u001b[0m | \u001b[95m 0.6532  \u001b[0m | \u001b[95m 3.734   \u001b[0m | \u001b[95m 210.6   \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m-3.859   \u001b[0m | \u001b[0m 0.9389  \u001b[0m | \u001b[0m 16.98   \u001b[0m | \u001b[0m 219.2   \u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m-3.747   \u001b[0m | \u001b[0m 0.8091  \u001b[0m | \u001b[0m 9.287   \u001b[0m | \u001b[0m 270.4   \u001b[0m |\n",
      "| \u001b[95m 11      \u001b[0m | \u001b[95m-3.667   \u001b[0m | \u001b[95m 0.999   \u001b[0m | \u001b[95m 2.0     \u001b[0m | \u001b[95m 334.9   \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m-3.668   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 142.0   \u001b[0m |\n",
      "| \u001b[0m 13      \u001b[0m | \u001b[0m-3.963   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 25.0    \u001b[0m | \u001b[0m 325.9   \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m-3.968   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 25.0    \u001b[0m | \u001b[0m 135.0   \u001b[0m |\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m-3.697   \u001b[0m | \u001b[0m 0.1493  \u001b[0m | \u001b[0m 2.014   \u001b[0m | \u001b[0m 399.6   \u001b[0m |\n",
      "| \u001b[95m 16      \u001b[0m | \u001b[95m-3.666   \u001b[0m | \u001b[95m 0.999   \u001b[0m | \u001b[95m 2.0     \u001b[0m | \u001b[95m 366.8   \u001b[0m |\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m-3.67    \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 301.2   \u001b[0m |\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m-3.668   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 258.9   \u001b[0m |\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m-3.668   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 171.1   \u001b[0m |\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m-3.678   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 100.0   \u001b[0m |\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m-3.676   \u001b[0m | \u001b[0m 0.88    \u001b[0m | \u001b[0m 2.051   \u001b[0m | \u001b[0m 121.6   \u001b[0m |\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m-3.668   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 227.6   \u001b[0m |\n",
      "| \u001b[0m 23      \u001b[0m | \u001b[0m-3.739   \u001b[0m | \u001b[0m 0.1363  \u001b[0m | \u001b[0m 2.138   \u001b[0m | \u001b[0m 382.2   \u001b[0m |\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m-3.667   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 350.7   \u001b[0m |\n",
      "| \u001b[0m 25      \u001b[0m | \u001b[0m-3.667   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 190.2   \u001b[0m |\n",
      "=============================================================\n",
      "Final result: {'target': -3.6659180133310714, 'params': {'max_features': 0.999, 'min_samples_split': 2.0, 'n_estimators': 366.84350004540295}}\n"
     ]
    }
   ],
   "source": [
    "print(Colours.yellow(\"--- Optimizing Extra Trees ---\"))\n",
    "optimize_etc(train, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "etc = ExtraTreesRegressor(n_estimators=int(366.84350004540295),\n",
    "                         min_samples_split=int(2.0),\n",
    "                         max_features=0.999,\n",
    "                         n_jobs = -1,\n",
    "                         random_state = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyperparameter tuning of RandomForest Regressor using Bayesian optimization\n",
    "\n",
    "def rfc_cv(n_estimators, min_samples_split, max_features, data, targets):\n",
    "    estimator = RandomForestRegressor(\n",
    "        n_estimators=n_estimators,\n",
    "        min_samples_split=min_samples_split,\n",
    "        max_features=max_features,\n",
    "        random_state=2,\n",
    "        n_jobs = -1\n",
    "    )\n",
    "    cval = cross_val_score(estimator, data, targets,\n",
    "                           scoring=rmse, cv=5)\n",
    "    return cval.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_rfc(data, targets):\n",
    "    def rfc_crossval(n_estimators, min_samples_split, max_features):\n",
    "        return rfc_cv(\n",
    "            n_estimators=int(n_estimators),\n",
    "            min_samples_split=int(min_samples_split),\n",
    "            max_features=max(min(max_features, 0.999), 1e-3),\n",
    "            data=data,\n",
    "            targets=targets,\n",
    "        )\n",
    "\n",
    "    optimizer = BayesianOptimization(\n",
    "        f=rfc_crossval,\n",
    "        pbounds={\n",
    "            \"n_estimators\": (100, 450),\n",
    "            \"min_samples_split\": (2, 25),\n",
    "            \"max_features\": (0.1, 0.999),\n",
    "        },\n",
    "        random_state=1234,\n",
    "        verbose=2\n",
    "    )\n",
    "    optimizer.maximize(n_iter=15, init_points=10)\n",
    "\n",
    "    print(\"Final result:\", optimizer.max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m--- Optimizing Random Forest ---\u001b[0m\n",
      "|   iter    |  target   | max_fe... | min_sa... | n_esti... |\n",
      "-------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m-3.945   \u001b[0m | \u001b[0m 0.2722  \u001b[0m | \u001b[0m 16.31   \u001b[0m | \u001b[0m 253.2   \u001b[0m |\n",
      "| \u001b[0m 2       \u001b[0m | \u001b[0m-3.996   \u001b[0m | \u001b[0m 0.806   \u001b[0m | \u001b[0m 19.94   \u001b[0m | \u001b[0m 195.4   \u001b[0m |\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m-3.975   \u001b[0m | \u001b[0m 0.3485  \u001b[0m | \u001b[0m 20.44   \u001b[0m | \u001b[0m 435.3   \u001b[0m |\n",
      "| \u001b[95m 4       \u001b[0m | \u001b[95m-3.917   \u001b[0m | \u001b[95m 0.8875  \u001b[0m | \u001b[95m 10.23   \u001b[0m | \u001b[95m 275.3   \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m-3.979   \u001b[0m | \u001b[0m 0.7144  \u001b[0m | \u001b[0m 18.39   \u001b[0m | \u001b[0m 229.6   \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m-3.929   \u001b[0m | \u001b[0m 0.6045  \u001b[0m | \u001b[0m 13.57   \u001b[0m | \u001b[0m 104.8   \u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m-4.01    \u001b[0m | \u001b[0m 0.7948  \u001b[0m | \u001b[0m 22.3    \u001b[0m | \u001b[0m 227.7   \u001b[0m |\n",
      "| \u001b[95m 8       \u001b[0m | \u001b[95m-3.845   \u001b[0m | \u001b[95m 0.6532  \u001b[0m | \u001b[95m 3.734   \u001b[0m | \u001b[95m 229.1   \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m-3.976   \u001b[0m | \u001b[0m 0.9389  \u001b[0m | \u001b[0m 16.98   \u001b[0m | \u001b[0m 239.0   \u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m-3.897   \u001b[0m | \u001b[0m 0.8091  \u001b[0m | \u001b[0m 9.287   \u001b[0m | \u001b[0m 298.8   \u001b[0m |\n",
      "| \u001b[95m 11      \u001b[0m | \u001b[95m-3.833   \u001b[0m | \u001b[95m 0.1     \u001b[0m | \u001b[95m 2.0     \u001b[0m | \u001b[95m 377.9   \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m-3.854   \u001b[0m | \u001b[0m 0.8123  \u001b[0m | \u001b[0m 4.699   \u001b[0m | \u001b[0m 377.9   \u001b[0m |\n",
      "| \u001b[95m 13      \u001b[0m | \u001b[95m-3.812   \u001b[0m | \u001b[95m 0.2212  \u001b[0m | \u001b[95m 2.083   \u001b[0m | \u001b[95m 143.6   \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m-3.855   \u001b[0m | \u001b[0m 0.8866  \u001b[0m | \u001b[0m 2.291   \u001b[0m | \u001b[0m 448.7   \u001b[0m |\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m-3.854   \u001b[0m | \u001b[0m 0.1185  \u001b[0m | \u001b[0m 2.103   \u001b[0m | \u001b[0m 103.0   \u001b[0m |\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m-3.836   \u001b[0m | \u001b[0m 0.1011  \u001b[0m | \u001b[0m 2.246   \u001b[0m | \u001b[0m 340.0   \u001b[0m |\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m-3.841   \u001b[0m | \u001b[0m 0.117   \u001b[0m | \u001b[0m 2.089   \u001b[0m | \u001b[0m 178.2   \u001b[0m |\n",
      "| \u001b[95m 18      \u001b[0m | \u001b[95m-3.808   \u001b[0m | \u001b[95m 0.1725  \u001b[0m | \u001b[95m 2.202   \u001b[0m | \u001b[95m 419.1   \u001b[0m |\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m-4.06    \u001b[0m | \u001b[0m 0.1471  \u001b[0m | \u001b[0m 24.95   \u001b[0m | \u001b[0m 331.8   \u001b[0m |\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m-3.824   \u001b[0m | \u001b[0m 0.4875  \u001b[0m | \u001b[0m 2.184   \u001b[0m | \u001b[0m 125.5   \u001b[0m |\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m-3.833   \u001b[0m | \u001b[0m 0.1219  \u001b[0m | \u001b[0m 2.154   \u001b[0m | \u001b[0m 402.7   \u001b[0m |\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m-3.84    \u001b[0m | \u001b[0m 0.1107  \u001b[0m | \u001b[0m 2.097   \u001b[0m | \u001b[0m 206.9   \u001b[0m |\n",
      "| \u001b[0m 23      \u001b[0m | \u001b[0m-3.837   \u001b[0m | \u001b[0m 0.1184  \u001b[0m | \u001b[0m 2.213   \u001b[0m | \u001b[0m 261.5   \u001b[0m |\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m-3.828   \u001b[0m | \u001b[0m 0.6329  \u001b[0m | \u001b[0m 2.074   \u001b[0m | \u001b[0m 433.7   \u001b[0m |\n",
      "| \u001b[95m 25      \u001b[0m | \u001b[95m-3.806   \u001b[0m | \u001b[95m 0.1843  \u001b[0m | \u001b[95m 2.153   \u001b[0m | \u001b[95m 310.0   \u001b[0m |\n",
      "=============================================================\n",
      "Final result: {'target': -3.8064683214054726, 'params': {'max_features': 0.18433272597322473, 'min_samples_split': 2.1527588902394075, 'n_estimators': 309.9622556931372}}\n"
     ]
    }
   ],
   "source": [
    "print(Colours.green(\"--- Optimizing Random Forest ---\"))\n",
    "optimize_rfc(train, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc = RandomForestRegressor(n_estimators = int(309.9622556931372),\n",
    "                           min_samples_split=int(2.1527588902394075),\n",
    "                           max_features = 0.18433272597322473,\n",
    "                           n_jobs = -1, \n",
    "                           random_state = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyperparameter tuning of LightGBM Regressor using Bayesian optimization\n",
    "\n",
    "def lgb_cv(n_estimators, num_leaves, min_child_samples, subsample, data, targets):\n",
    "    estimator = LGBMRegressor(\n",
    "        n_estimators=n_estimators,\n",
    "        num_leaves = num_leaves,\n",
    "        min_child_samples=min_child_samples,\n",
    "        subsample = subsample,\n",
    "        random_state = 2\n",
    "    )\n",
    "    cval = cross_val_score(estimator, data, targets,\n",
    "                           scoring=rmse, cv=5)\n",
    "    return cval.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_lgb(data, targets):\n",
    "    def lgb_crossval(n_estimators, num_leaves, min_child_samples, subsample):\n",
    "        return lgb_cv(\n",
    "            n_estimators=int(n_estimators),\n",
    "            num_leaves = int(num_leaves),\n",
    "            min_child_samples=int(min_child_samples),\n",
    "            subsample=subsample,\n",
    "            data=data,\n",
    "            targets=targets,\n",
    "        )\n",
    "\n",
    "    optimizer = BayesianOptimization(\n",
    "        f=lgb_crossval,\n",
    "        pbounds={\n",
    "            \"n_estimators\": (200,500),\n",
    "            \"num_leaves\": (30,80),\n",
    "            \"min_child_samples\": (5,30),\n",
    "            \"subsample\": (0.6,1.0)\n",
    "        },\n",
    "        random_state=1234,\n",
    "        verbose=2\n",
    "    )\n",
    "    optimizer.maximize(n_iter=25, init_points=20)\n",
    "\n",
    "    print(\"Final result:\", optimizer.max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[94m--- Optimizing Light GBM ---\u001b[0m\n",
      "|   iter    |  target   | min_ch... | n_esti... | num_le... | subsample |\n",
      "-------------------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m-3.855   \u001b[0m | \u001b[0m 9.788   \u001b[0m | \u001b[0m 386.6   \u001b[0m | \u001b[0m 51.89   \u001b[0m | \u001b[0m 0.9141  \u001b[0m |\n",
      "| \u001b[0m 2       \u001b[0m | \u001b[0m-3.89    \u001b[0m | \u001b[0m 24.5    \u001b[0m | \u001b[0m 281.8   \u001b[0m | \u001b[0m 43.82   \u001b[0m | \u001b[0m 0.9207  \u001b[0m |\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m-3.857   \u001b[0m | \u001b[0m 28.95   \u001b[0m | \u001b[0m 462.8   \u001b[0m | \u001b[0m 47.89   \u001b[0m | \u001b[0m 0.8004  \u001b[0m |\n",
      "| \u001b[95m 4       \u001b[0m | \u001b[95m-3.848   \u001b[0m | \u001b[95m 22.09   \u001b[0m | \u001b[95m 413.8   \u001b[0m | \u001b[95m 48.51   \u001b[0m | \u001b[95m 0.8245  \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m-3.854   \u001b[0m | \u001b[0m 17.58   \u001b[0m | \u001b[0m 204.1   \u001b[0m | \u001b[0m 68.64   \u001b[0m | \u001b[0m 0.9531  \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m-3.876   \u001b[0m | \u001b[0m 14.12   \u001b[0m | \u001b[0m 384.6   \u001b[0m | \u001b[0m 33.77   \u001b[0m | \u001b[0m 0.7475  \u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m-3.879   \u001b[0m | \u001b[0m 28.33   \u001b[0m | \u001b[0m 395.4   \u001b[0m | \u001b[0m 49.86   \u001b[0m | \u001b[0m 0.9155  \u001b[0m |\n",
      "| \u001b[95m 8       \u001b[0m | \u001b[95m-3.838   \u001b[0m | \u001b[95m 12.92   \u001b[0m | \u001b[95m 370.4   \u001b[0m | \u001b[95m 73.46   \u001b[0m | \u001b[95m 0.7745  \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m-3.859   \u001b[0m | \u001b[0m 25.05   \u001b[0m | \u001b[0m 243.1   \u001b[0m | \u001b[0m 65.21   \u001b[0m | \u001b[0m 0.8818  \u001b[0m |\n",
      "| \u001b[95m 10      \u001b[0m | \u001b[95m-3.831   \u001b[0m | \u001b[95m 10.47   \u001b[0m | \u001b[95m 477.5   \u001b[0m | \u001b[95m 52.11   \u001b[0m | \u001b[95m 0.9637  \u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m-3.927   \u001b[0m | \u001b[0m 6.495   \u001b[0m | \u001b[0m 255.3   \u001b[0m | \u001b[0m 32.37   \u001b[0m | \u001b[0m 0.87    \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m-3.895   \u001b[0m | \u001b[0m 19.87   \u001b[0m | \u001b[0m 360.0   \u001b[0m | \u001b[0m 32.17   \u001b[0m | \u001b[0m 0.8246  \u001b[0m |\n",
      "| \u001b[0m 13      \u001b[0m | \u001b[0m-3.897   \u001b[0m | \u001b[0m 13.24   \u001b[0m | \u001b[0m 350.9   \u001b[0m | \u001b[0m 35.59   \u001b[0m | \u001b[0m 0.8429  \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m-3.863   \u001b[0m | \u001b[0m 19.15   \u001b[0m | \u001b[0m 202.0   \u001b[0m | \u001b[0m 60.87   \u001b[0m | \u001b[0m 0.9648  \u001b[0m |\n",
      "| \u001b[95m 15      \u001b[0m | \u001b[95m-3.826   \u001b[0m | \u001b[95m 24.76   \u001b[0m | \u001b[95m 497.6   \u001b[0m | \u001b[95m 77.94   \u001b[0m | \u001b[95m 0.9168  \u001b[0m |\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m-3.843   \u001b[0m | \u001b[0m 12.13   \u001b[0m | \u001b[0m 387.5   \u001b[0m | \u001b[0m 53.9    \u001b[0m | \u001b[0m 0.6783  \u001b[0m |\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m-3.878   \u001b[0m | \u001b[0m 14.56   \u001b[0m | \u001b[0m 216.2   \u001b[0m | \u001b[0m 52.58   \u001b[0m | \u001b[0m 0.9928  \u001b[0m |\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m-3.84    \u001b[0m | \u001b[0m 8.099   \u001b[0m | \u001b[0m 235.8   \u001b[0m | \u001b[0m 66.93   \u001b[0m | \u001b[0m 0.8349  \u001b[0m |\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m-3.885   \u001b[0m | \u001b[0m 16.79   \u001b[0m | \u001b[0m 232.1   \u001b[0m | \u001b[0m 41.46   \u001b[0m | \u001b[0m 0.96    \u001b[0m |\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m-3.88    \u001b[0m | \u001b[0m 15.42   \u001b[0m | \u001b[0m 360.8   \u001b[0m | \u001b[0m 30.31   \u001b[0m | \u001b[0m 0.7203  \u001b[0m |\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m-3.83    \u001b[0m | \u001b[0m 6.388   \u001b[0m | \u001b[0m 453.8   \u001b[0m | \u001b[0m 79.37   \u001b[0m | \u001b[0m 0.8022  \u001b[0m |\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m-3.883   \u001b[0m | \u001b[0m 21.3    \u001b[0m | \u001b[0m 498.9   \u001b[0m | \u001b[0m 30.36   \u001b[0m | \u001b[0m 0.6823  \u001b[0m |\n",
      "| \u001b[0m 23      \u001b[0m | \u001b[0m-3.842   \u001b[0m | \u001b[0m 5.34    \u001b[0m | \u001b[0m 294.8   \u001b[0m | \u001b[0m 79.25   \u001b[0m | \u001b[0m 0.6585  \u001b[0m |\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m-3.833   \u001b[0m | \u001b[0m 27.24   \u001b[0m | \u001b[0m 412.1   \u001b[0m | \u001b[0m 79.23   \u001b[0m | \u001b[0m 0.6     \u001b[0m |\n",
      "| \u001b[0m 25      \u001b[0m | \u001b[0m-3.836   \u001b[0m | \u001b[0m 6.682   \u001b[0m | \u001b[0m 495.8   \u001b[0m | \u001b[0m 77.99   \u001b[0m | \u001b[0m 0.6828  \u001b[0m |\n",
      "| \u001b[0m 26      \u001b[0m | \u001b[0m-3.95    \u001b[0m | \u001b[0m 29.42   \u001b[0m | \u001b[0m 200.1   \u001b[0m | \u001b[0m 30.41   \u001b[0m | \u001b[0m 0.9826  \u001b[0m |\n",
      "| \u001b[0m 27      \u001b[0m | \u001b[0m-3.836   \u001b[0m | \u001b[0m 5.448   \u001b[0m | \u001b[0m 405.6   \u001b[0m | \u001b[0m 79.54   \u001b[0m | \u001b[0m 0.6633  \u001b[0m |\n",
      "| \u001b[0m 28      \u001b[0m | \u001b[0m-3.896   \u001b[0m | \u001b[0m 5.449   \u001b[0m | \u001b[0m 460.7   \u001b[0m | \u001b[0m 30.3    \u001b[0m | \u001b[0m 0.6282  \u001b[0m |\n",
      "| \u001b[0m 29      \u001b[0m | \u001b[0m-3.836   \u001b[0m | \u001b[0m 28.8    \u001b[0m | \u001b[0m 309.2   \u001b[0m | \u001b[0m 79.8    \u001b[0m | \u001b[0m 0.6058  \u001b[0m |\n",
      "| \u001b[0m 30      \u001b[0m | \u001b[0m-3.848   \u001b[0m | \u001b[0m 5.633   \u001b[0m | \u001b[0m 200.1   \u001b[0m | \u001b[0m 73.61   \u001b[0m | \u001b[0m 0.9991  \u001b[0m |\n",
      "| \u001b[0m 31      \u001b[0m | \u001b[0m-3.838   \u001b[0m | \u001b[0m 29.26   \u001b[0m | \u001b[0m 467.2   \u001b[0m | \u001b[0m 79.73   \u001b[0m | \u001b[0m 0.6445  \u001b[0m |\n",
      "| \u001b[0m 32      \u001b[0m | \u001b[0m-3.836   \u001b[0m | \u001b[0m 16.53   \u001b[0m | \u001b[0m 271.7   \u001b[0m | \u001b[0m 78.99   \u001b[0m | \u001b[0m 0.7329  \u001b[0m |\n",
      "| \u001b[0m 33      \u001b[0m | \u001b[0m-3.85    \u001b[0m | \u001b[0m 13.36   \u001b[0m | \u001b[0m 440.8   \u001b[0m | \u001b[0m 65.19   \u001b[0m | \u001b[0m 0.6029  \u001b[0m |\n",
      "| \u001b[0m 34      \u001b[0m | \u001b[0m-3.834   \u001b[0m | \u001b[0m 28.75   \u001b[0m | \u001b[0m 381.8   \u001b[0m | \u001b[0m 79.7    \u001b[0m | \u001b[0m 0.9708  \u001b[0m |\n",
      "| \u001b[0m 35      \u001b[0m | \u001b[0m-3.861   \u001b[0m | \u001b[0m 28.66   \u001b[0m | \u001b[0m 497.9   \u001b[0m | \u001b[0m 59.61   \u001b[0m | \u001b[0m 0.6499  \u001b[0m |\n",
      "| \u001b[95m 36      \u001b[0m | \u001b[95m-3.818   \u001b[0m | \u001b[95m 13.58   \u001b[0m | \u001b[95m 478.4   \u001b[0m | \u001b[95m 79.3    \u001b[0m | \u001b[95m 0.999   \u001b[0m |\n",
      "| \u001b[0m 37      \u001b[0m | \u001b[0m-3.822   \u001b[0m | \u001b[0m 8.044   \u001b[0m | \u001b[0m 230.4   \u001b[0m | \u001b[0m 79.97   \u001b[0m | \u001b[0m 0.6769  \u001b[0m |\n",
      "| \u001b[0m 38      \u001b[0m | \u001b[0m-3.826   \u001b[0m | \u001b[0m 14.88   \u001b[0m | \u001b[0m 329.5   \u001b[0m | \u001b[0m 79.81   \u001b[0m | \u001b[0m 0.6654  \u001b[0m |\n",
      "| \u001b[0m 39      \u001b[0m | \u001b[0m-3.85    \u001b[0m | \u001b[0m 29.44   \u001b[0m | \u001b[0m 200.9   \u001b[0m | \u001b[0m 79.44   \u001b[0m | \u001b[0m 0.6483  \u001b[0m |\n",
      "| \u001b[0m 40      \u001b[0m | \u001b[0m-3.848   \u001b[0m | \u001b[0m 5.394   \u001b[0m | \u001b[0m 255.6   \u001b[0m | \u001b[0m 79.67   \u001b[0m | \u001b[0m 0.7582  \u001b[0m |\n",
      "| \u001b[0m 41      \u001b[0m | \u001b[0m-3.835   \u001b[0m | \u001b[0m 18.47   \u001b[0m | \u001b[0m 392.9   \u001b[0m | \u001b[0m 78.71   \u001b[0m | \u001b[0m 0.7088  \u001b[0m |\n",
      "| \u001b[0m 42      \u001b[0m | \u001b[0m-3.84    \u001b[0m | \u001b[0m 29.1    \u001b[0m | \u001b[0m 343.2   \u001b[0m | \u001b[0m 79.0    \u001b[0m | \u001b[0m 0.6524  \u001b[0m |\n",
      "| \u001b[0m 43      \u001b[0m | \u001b[0m-3.823   \u001b[0m | \u001b[0m 5.003   \u001b[0m | \u001b[0m 470.1   \u001b[0m | \u001b[0m 70.86   \u001b[0m | \u001b[0m 0.7334  \u001b[0m |\n",
      "| \u001b[0m 44      \u001b[0m | \u001b[0m-3.84    \u001b[0m | \u001b[0m 13.01   \u001b[0m | \u001b[0m 409.1   \u001b[0m | \u001b[0m 64.57   \u001b[0m | \u001b[0m 0.6169  \u001b[0m |\n",
      "| \u001b[0m 45      \u001b[0m | \u001b[0m-3.837   \u001b[0m | \u001b[0m 28.75   \u001b[0m | \u001b[0m 279.6   \u001b[0m | \u001b[0m 79.63   \u001b[0m | \u001b[0m 0.8306  \u001b[0m |\n",
      "=========================================================================\n",
      "Final result: {'target': -3.818232371399062, 'params': {'min_child_samples': 13.577585500929393, 'n_estimators': 478.3739357476376, 'num_leaves': 79.29868320714608, 'subsample': 0.9989768631907607}}\n"
     ]
    }
   ],
   "source": [
    "print(Colours.blue(\"--- Optimizing Light GBM ---\"))\n",
    "optimize_lgb(train, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb = LGBMRegressor(\n",
    "        n_estimators=int(478.3739357476376),\n",
    "        num_leaves = int(79.29868320714608),\n",
    "        min_child_samples=int(13.577585500929393),\n",
    "        subsample = 1.0,\n",
    "        random_state = 2\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyperparameter tuning of XGBoost Regressor using Bayesian optimization\n",
    "\n",
    "def xgb_cv(n_estimators, max_depth, gamma, min_child_weight, subsample, data, targets):\n",
    "    estimator = XGBRegressor(\n",
    "        n_estimators=n_estimators,\n",
    "        max_depth = max_depth,\n",
    "        gamma = gamma,\n",
    "        min_child_weight=min_child_weight,\n",
    "        subsample = subsample,\n",
    "        random_state = 2,\n",
    "    )\n",
    "    cval = cross_val_score(estimator, data, targets,\n",
    "                           scoring=rmse, cv=5)\n",
    "    return cval.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_xgb(data, targets):\n",
    "    def xgb_crossval(n_estimators, max_depth, gamma, min_child_weight, subsample):\n",
    "        return xgb_cv(\n",
    "            n_estimators=int(n_estimators),\n",
    "            max_depth = int(max_depth),\n",
    "            gamma = gamma,\n",
    "            min_child_weight=min_child_weight,\n",
    "            subsample=subsample,\n",
    "            data=data,\n",
    "            targets=targets,\n",
    "        )\n",
    "\n",
    "    optimizer = BayesianOptimization(\n",
    "        f=xgb_crossval,\n",
    "        pbounds={\n",
    "            \"n_estimators\": (200, 500),\n",
    "            \"max_depth\": (6,15),\n",
    "            \"gamma\": (0,10),\n",
    "            \"min_child_weight\": (0,10),\n",
    "            \"subsample\": (0.8,1.0)\n",
    "        },\n",
    "        random_state=1234,\n",
    "        verbose=2\n",
    "    )\n",
    "    optimizer.maximize(n_iter=25, init_points=20)\n",
    "\n",
    "    print(\"Final result:\", optimizer.max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[91m--- Optimizing XGBoost ---\u001b[0m\n",
      "|   iter    |  target   |   gamma   | max_depth | min_ch... | n_esti... | subsample |\n",
      "-------------------------------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m-4.058   \u001b[0m | \u001b[0m 1.915   \u001b[0m | \u001b[0m 11.6    \u001b[0m | \u001b[0m 4.377   \u001b[0m | \u001b[0m 435.6   \u001b[0m | \u001b[0m 0.956   \u001b[0m |\n",
      "| \u001b[95m 2       \u001b[0m | \u001b[95m-4.044   \u001b[0m | \u001b[95m 2.726   \u001b[0m | \u001b[95m 8.488   \u001b[0m | \u001b[95m 8.019   \u001b[0m | \u001b[95m 487.4   \u001b[0m | \u001b[95m 0.9752  \u001b[0m |\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m-4.085   \u001b[0m | \u001b[0m 3.578   \u001b[0m | \u001b[0m 10.51   \u001b[0m | \u001b[0m 6.835   \u001b[0m | \u001b[0m 413.8   \u001b[0m | \u001b[0m 0.8741  \u001b[0m |\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m-4.074   \u001b[0m | \u001b[0m 5.612   \u001b[0m | \u001b[0m 10.53   \u001b[0m | \u001b[0m 0.1377  \u001b[0m | \u001b[0m 431.8   \u001b[0m | \u001b[0m 0.9765  \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m-4.077   \u001b[0m | \u001b[0m 3.649   \u001b[0m | \u001b[0m 11.54   \u001b[0m | \u001b[0m 0.7538  \u001b[0m | \u001b[0m 310.6   \u001b[0m | \u001b[0m 0.9866  \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m-4.095   \u001b[0m | \u001b[0m 6.514   \u001b[0m | \u001b[0m 9.575   \u001b[0m | \u001b[0m 7.887   \u001b[0m | \u001b[0m 295.1   \u001b[0m | \u001b[0m 0.9136  \u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m-4.078   \u001b[0m | \u001b[0m 8.691   \u001b[0m | \u001b[0m 9.926   \u001b[0m | \u001b[0m 8.021   \u001b[0m | \u001b[0m 243.1   \u001b[0m | \u001b[0m 0.9409  \u001b[0m |\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m-4.05    \u001b[0m | \u001b[0m 7.046   \u001b[0m | \u001b[0m 7.969   \u001b[0m | \u001b[0m 9.249   \u001b[0m | \u001b[0m 332.6   \u001b[0m | \u001b[0m 0.9819  \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m-4.108   \u001b[0m | \u001b[0m 0.5981  \u001b[0m | \u001b[0m 7.659   \u001b[0m | \u001b[0m 0.4736  \u001b[0m | \u001b[0m 402.5   \u001b[0m | \u001b[0m 0.9189  \u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m-4.143   \u001b[0m | \u001b[0m 5.333   \u001b[0m | \u001b[0m 6.39    \u001b[0m | \u001b[0m 5.614   \u001b[0m | \u001b[0m 298.9   \u001b[0m | \u001b[0m 0.9006  \u001b[0m |\n",
      "| \u001b[95m 11      \u001b[0m | \u001b[95m-4.03    \u001b[0m | \u001b[95m 1.119   \u001b[0m | \u001b[95m 11.46   \u001b[0m | \u001b[95m 5.659   \u001b[0m | \u001b[95m 202.0   \u001b[0m | \u001b[95m 0.9235  \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m-4.051   \u001b[0m | \u001b[0m 9.121   \u001b[0m | \u001b[0m 13.11   \u001b[0m | \u001b[0m 9.921   \u001b[0m | \u001b[0m 487.6   \u001b[0m | \u001b[0m 0.9584  \u001b[0m |\n",
      "| \u001b[0m 13      \u001b[0m | \u001b[0m-4.071   \u001b[0m | \u001b[0m 2.853   \u001b[0m | \u001b[0m 11.62   \u001b[0m | \u001b[0m 4.781   \u001b[0m | \u001b[0m 258.7   \u001b[0m | \u001b[0m 0.8765  \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m-4.068   \u001b[0m | \u001b[0m 0.5387  \u001b[0m | \u001b[0m 10.06   \u001b[0m | \u001b[0m 9.82    \u001b[0m | \u001b[0m 237.2   \u001b[0m | \u001b[0m 0.8239  \u001b[0m |\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m-4.082   \u001b[0m | \u001b[0m 7.385   \u001b[0m | \u001b[0m 11.29   \u001b[0m | \u001b[0m 4.716   \u001b[0m | \u001b[0m 232.1   \u001b[0m | \u001b[0m 0.8458  \u001b[0m |\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m-4.112   \u001b[0m | \u001b[0m 9.0     \u001b[0m | \u001b[0m 9.751   \u001b[0m | \u001b[0m 5.359   \u001b[0m | \u001b[0m 201.9   \u001b[0m | \u001b[0m 0.8601  \u001b[0m |\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m-4.054   \u001b[0m | \u001b[0m 4.369   \u001b[0m | \u001b[0m 11.51   \u001b[0m | \u001b[0m 9.182   \u001b[0m | \u001b[0m 387.7   \u001b[0m | \u001b[0m 0.9412  \u001b[0m |\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m-4.047   \u001b[0m | \u001b[0m 1.498   \u001b[0m | \u001b[0m 12.71   \u001b[0m | \u001b[0m 8.31    \u001b[0m | \u001b[0m 390.1   \u001b[0m | \u001b[0m 0.8877  \u001b[0m |\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m-4.042   \u001b[0m | \u001b[0m 1.526   \u001b[0m | \u001b[0m 11.12   \u001b[0m | \u001b[0m 5.282   \u001b[0m | \u001b[0m 485.4   \u001b[0m | \u001b[0m 0.8961  \u001b[0m |\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m-4.062   \u001b[0m | \u001b[0m 5.026   \u001b[0m | \u001b[0m 10.83   \u001b[0m | \u001b[0m 8.192   \u001b[0m | \u001b[0m 217.1   \u001b[0m | \u001b[0m 0.9339  \u001b[0m |\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m-4.073   \u001b[0m | \u001b[0m 9.958   \u001b[0m | \u001b[0m 14.5    \u001b[0m | \u001b[0m 1.311   \u001b[0m | \u001b[0m 355.5   \u001b[0m | \u001b[0m 0.8954  \u001b[0m |\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m-4.08    \u001b[0m | \u001b[0m 7.127   \u001b[0m | \u001b[0m 6.407   \u001b[0m | \u001b[0m 0.5419  \u001b[0m | \u001b[0m 499.4   \u001b[0m | \u001b[0m 0.9596  \u001b[0m |\n",
      "| \u001b[0m 23      \u001b[0m | \u001b[0m-4.073   \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 338.4   \u001b[0m | \u001b[0m 0.8038  \u001b[0m |\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m-4.058   \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 500.0   \u001b[0m | \u001b[0m 0.8003  \u001b[0m |\n",
      "| \u001b[0m 25      \u001b[0m | \u001b[0m-4.094   \u001b[0m | \u001b[0m 4.34    \u001b[0m | \u001b[0m 14.62   \u001b[0m | \u001b[0m 9.908   \u001b[0m | \u001b[0m 210.4   \u001b[0m | \u001b[0m 0.8972  \u001b[0m |\n",
      "| \u001b[0m 26      \u001b[0m | \u001b[0m-4.04    \u001b[0m | \u001b[0m 9.308   \u001b[0m | \u001b[0m 14.12   \u001b[0m | \u001b[0m 9.868   \u001b[0m | \u001b[0m 323.6   \u001b[0m | \u001b[0m 0.9963  \u001b[0m |\n",
      "| \u001b[0m 27      \u001b[0m | \u001b[0m-4.108   \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 6.0     \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 457.7   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "| \u001b[0m 28      \u001b[0m | \u001b[0m-4.136   \u001b[0m | \u001b[0m 9.816   \u001b[0m | \u001b[0m 14.74   \u001b[0m | \u001b[0m 0.207   \u001b[0m | \u001b[0m 276.9   \u001b[0m | \u001b[0m 0.9849  \u001b[0m |\n",
      "| \u001b[0m 29      \u001b[0m | \u001b[0m-4.112   \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 6.0     \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 200.0   \u001b[0m | \u001b[0m 0.8     \u001b[0m |\n",
      "| \u001b[0m 30      \u001b[0m | \u001b[0m-4.077   \u001b[0m | \u001b[0m 0.3101  \u001b[0m | \u001b[0m 11.64   \u001b[0m | \u001b[0m 0.2621  \u001b[0m | \u001b[0m 210.2   \u001b[0m | \u001b[0m 0.9624  \u001b[0m |\n",
      "| \u001b[0m 31      \u001b[0m | \u001b[0m-4.081   \u001b[0m | \u001b[0m 0.2288  \u001b[0m | \u001b[0m 13.88   \u001b[0m | \u001b[0m 3.747   \u001b[0m | \u001b[0m 202.6   \u001b[0m | \u001b[0m 0.8501  \u001b[0m |\n",
      "| \u001b[0m 32      \u001b[0m | \u001b[0m-4.094   \u001b[0m | \u001b[0m 0.04106 \u001b[0m | \u001b[0m 6.1     \u001b[0m | \u001b[0m 4.463   \u001b[0m | \u001b[0m 363.0   \u001b[0m | \u001b[0m 0.9767  \u001b[0m |\n",
      "| \u001b[0m 33      \u001b[0m | \u001b[0m-4.148   \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 397.0   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "| \u001b[0m 34      \u001b[0m | \u001b[0m-4.183   \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 458.7   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "| \u001b[0m 35      \u001b[0m | \u001b[0m-4.076   \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 436.3   \u001b[0m | \u001b[0m 0.8     \u001b[0m |\n",
      "| \u001b[0m 36      \u001b[0m | \u001b[0m-4.108   \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 6.0     \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 270.1   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "| \u001b[0m 37      \u001b[0m | \u001b[0m-4.076   \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 6.0     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 244.7   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "| \u001b[95m 38      \u001b[0m | \u001b[95m-4.024   \u001b[0m | \u001b[95m 0.0     \u001b[0m | \u001b[95m 15.0    \u001b[0m | \u001b[95m 10.0    \u001b[0m | \u001b[95m 281.2   \u001b[0m | \u001b[95m 1.0     \u001b[0m |\n",
      "| \u001b[0m 39      \u001b[0m | \u001b[0m-4.144   \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 332.4   \u001b[0m | \u001b[0m 0.8     \u001b[0m |\n",
      "| \u001b[0m 40      \u001b[0m | \u001b[0m-4.11    \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 6.0     \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 356.9   \u001b[0m | \u001b[0m 0.8     \u001b[0m |\n",
      "| \u001b[0m 41      \u001b[0m | \u001b[0m-4.128   \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 6.0     \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 430.0   \u001b[0m | \u001b[0m 0.8     \u001b[0m |\n",
      "| \u001b[95m 42      \u001b[0m | \u001b[95m-4.024   \u001b[0m | \u001b[95m 0.0     \u001b[0m | \u001b[95m 15.0    \u001b[0m | \u001b[95m 10.0    \u001b[0m | \u001b[95m 370.2   \u001b[0m | \u001b[95m 1.0     \u001b[0m |\n",
      "| \u001b[0m 43      \u001b[0m | \u001b[0m-4.109   \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 6.0     \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 318.0   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "| \u001b[0m 44      \u001b[0m | \u001b[0m-4.153   \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 6.0     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 476.8   \u001b[0m | \u001b[0m 0.8     \u001b[0m |\n",
      "| \u001b[0m 45      \u001b[0m | \u001b[0m-4.183   \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 377.1   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "=====================================================================================\n",
      "Final result: {'target': -4.0240748319454696, 'params': {'gamma': 0.0, 'max_depth': 15.0, 'min_child_weight': 10.0, 'n_estimators': 370.2077789353612, 'subsample': 1.0}}\n"
     ]
    }
   ],
   "source": [
    "print(Colours.red(\"--- Optimizing XGBoost ---\"))\n",
    "optimize_xgb(train, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb = XGBRegressor(\n",
    "        n_estimators=int(370.207778935361),\n",
    "        max_depth = int(15.0),\n",
    "        gamma = 0.0,\n",
    "        min_child_weight=10.0,\n",
    "        subsample = 1.0,\n",
    "        random_state = 2,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyperparameter tuning of Catboost Regressor using Bayesian optimization\n",
    "\n",
    "def cb_cv(n_estimators, depth,data, targets):\n",
    "    estimator = CatBoostRegressor(\n",
    "        n_estimators=n_estimators,\n",
    "#         learning_rate=learning_rate,\n",
    "        depth=depth,\n",
    "        random_state = 2,\n",
    "        verbose = 0,\n",
    "    )\n",
    "    cval = cross_val_score(estimator, data, targets,\n",
    "                           scoring=rmse, cv=5)\n",
    "    return cval.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_cb(data, targets):\n",
    "    def cb_crossval(n_estimators, depth):\n",
    "        return cb_cv(\n",
    "            n_estimators=int(n_estimators),\n",
    "#             learning_rate = learning_rate,\n",
    "            depth = int(depth),\n",
    "            data=data,\n",
    "            targets=targets,\n",
    "        )\n",
    "\n",
    "    optimizer = BayesianOptimization(\n",
    "        f=cb_crossval,\n",
    "        pbounds={\n",
    "            \"n_estimators\": (200, 600),\n",
    "#             \"learning_rate\": (0.01,10),\n",
    "            \"depth\": (4,16),\n",
    "        },\n",
    "        random_state=1234,\n",
    "        verbose=2\n",
    "    )\n",
    "    optimizer.maximize(n_iter=25, init_points=20)\n",
    "\n",
    "    print(\"Final result:\", optimizer.max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(Colours.bold(\"--- Optimizing Catboost ---\"))\n",
    "optimize_cb(train, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cb = CatBoostRegressor(n_estimators=309,\n",
    "                      depth = 13,\n",
    "                      verbose = 0,\n",
    "                      random_state = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyperparameter tuning of XGBoost RandomForest Regressor using Bayesian optimization\n",
    "\n",
    "def xgb_cv(n_estimators, max_depth, gamma, min_child_weight, subsample, data, targets):\n",
    "    estimator = XGBRFRegressor(\n",
    "        n_estimators=n_estimators,\n",
    "        max_depth = max_depth,\n",
    "        gamma = gamma,\n",
    "        min_child_weight=min_child_weight,\n",
    "        subsample = subsample,\n",
    "        random_state = 2,\n",
    "    )\n",
    "    cval = cross_val_score(estimator, data, targets,\n",
    "                           scoring=rmse, cv=5)\n",
    "    return cval.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_xgb(data, targets):\n",
    "    def xgb_crossval(n_estimators, max_depth, gamma, min_child_weight, subsample):\n",
    "        return xgb_cv(\n",
    "            n_estimators=int(n_estimators),\n",
    "            max_depth = int(max_depth),\n",
    "            gamma = gamma,\n",
    "            min_child_weight=min_child_weight,\n",
    "            subsample=subsample,\n",
    "            data=data,\n",
    "            targets=targets,\n",
    "        )\n",
    "\n",
    "    optimizer = BayesianOptimization(\n",
    "        f=xgb_crossval,\n",
    "        pbounds={\n",
    "            \"n_estimators\": (200, 500),\n",
    "            \"max_depth\": (6,15),\n",
    "            \"gamma\": (0,10),\n",
    "            \"min_child_weight\": (0,10),\n",
    "            \"subsample\": (0.8,1.0)\n",
    "        },\n",
    "        random_state=1234,\n",
    "        verbose=2\n",
    "    )\n",
    "    optimizer.maximize(n_iter=25, init_points=20)\n",
    "\n",
    "    print(\"Final result:\", optimizer.max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(Colours.red(\"--- Optimizing XGBoost RandomForest ---\"))\n",
    "optimize_xgb(train, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbrf = XGBRFRegressor(\n",
    "        n_estimators=int(463.8),\n",
    "        max_depth = int(15.0),\n",
    "        gamma = 10.0,\n",
    "        min_child_weight=0.0,\n",
    "        subsample = 0.8,\n",
    "        random_state = 2,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Stacking least correlated models\n",
    "\n",
    "estimators = [('etc', etc), ('rfc', rfc), ('xgb', xgb), ('lgb', lgb), ('xgbrf', xgbrf)]\n",
    "\n",
    "model = StackingRegressor(estimators=estimators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-3.5971750064096297"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = cross_val_score(model, train, target, cv = 5, scoring = rmse)\n",
    "\n",
    "scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StackingRegressor(cv=None,\n",
       "                  estimators=[('etc',\n",
       "                               ExtraTreesRegressor(bootstrap=False,\n",
       "                                                   ccp_alpha=0.0,\n",
       "                                                   criterion='mse',\n",
       "                                                   max_depth=None,\n",
       "                                                   max_features=0.999,\n",
       "                                                   max_leaf_nodes=None,\n",
       "                                                   max_samples=None,\n",
       "                                                   min_impurity_decrease=0.0,\n",
       "                                                   min_impurity_split=None,\n",
       "                                                   min_samples_leaf=1,\n",
       "                                                   min_samples_split=2,\n",
       "                                                   min_weight_fraction_leaf=0.0,\n",
       "                                                   n_estimators=366, n_jobs=-1,\n",
       "                                                   oob_score=False,\n",
       "                                                   random_...\n",
       "                                              max_delta_step=None, max_depth=15,\n",
       "                                              min_child_weight=0.0, missing=nan,\n",
       "                                              monotone_constraints=None,\n",
       "                                              n_estimators=463, n_jobs=None,\n",
       "                                              num_parallel_tree=None,\n",
       "                                              objective='reg:squarederror',\n",
       "                                              random_state=2, reg_alpha=None,\n",
       "                                              reg_lambda=1e-05,\n",
       "                                              scale_pos_weight=None,\n",
       "                                              subsample=0.8, tree_method=None,\n",
       "                                              validate_parameters=None,\n",
       "                                              verbosity=None))],\n",
       "                  final_estimator=None, n_jobs=None, passthrough=False,\n",
       "                  verbose=0)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Fitting model\n",
    "\n",
    "model.fit(train, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Predicting\n",
    "\n",
    "preds = model.predict(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('/run/media/devesh/Hard Disk/Projects/Machinehack-Melanoma Tumor Size Predicition/input/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub['tumor_size'] = preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating submission file\n",
    "\n",
    "sub.to_csv('ensemble_2.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
